
import numpy as np
import pandas as pd
import re, string
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns

pd.set_option('display.max_rows', 20)
pd.set_option('display.max_columns', None)
pd.set_option('max_colwidth', None)


def read_data(file, info=True):
    try:
        df = pd.read_csv(file)
        if info:
            print('\n', "="*10+"LOADING DATA"+"="*10)
            print("Data reading is complete!")

            print('\n', "="*10+"DIMENSIONS"+"="*10)
            print("Number of Observations = ", df.shape[0])
            print("Number of Columns = ", df.shape[1],'\n')
            
            print('\n', "="*10+"DTYPES"+"="*10)
            if len(df.select_dtypes("object").columns) > 0:
                print('Number of Object variables = ', 
                      len(df.select_dtypes("object").columns))
            if len(df.select_dtypes("integer").columns) > 0:
                print('Number of Integer variables = ', 
                      len(df.select_dtypes("integer").columns))
            if len(df.select_dtypes("float").columns) > 0:
                print('Number of Float variables = ', 
                      len(df.select_dtypes("float").columns))
            if len(df.select_dtypes("bool").columns) > 0:
                print('Number of Boolean variables = ', 
                      len(df.select_dtypes("bool").columns))
                
            print('\n', "="*10+"MISSING VALUES"+"="*10)
            print(np.where(df.isnull().values.any() == False, 
                           "No missing value! Proceed to next steps..."
                           , "Data includes missing value! Analyze missing values"))
            
        return df
    except:
        print('Data did not import!')


df = read_data('case_study_data.csv',info=True)
df.head(2)

 ==========LOADING DATA==========
Data reading is complete!

 ==========DIMENSIONS==========
Number of Observations =  268343
Number of Columns =  3 


 ==========DTYPES==========
Number of Object variables =  2
Number of Integer variables =  1

 ==========MISSING VALUES==========
No missing value! Proceed to next steps...


def plot_counts(df, var, axs):
    chart = sns.countplot(x=df[var],order = df[var].value_counts().index,ax=axs)
    chart.set_xticklabels(chart.get_xticklabels(),rotation=90)
    for p, label in zip(axs.patches, df[var].value_counts()):
        axs.annotate(label, (p.get_x()+0.1, p.get_height()+1));
        
fig, axs =plt.subplots(1,2, figsize=(15,5))
fig.suptitle('Distribution of Product Groups', weight='bold', size=16)

plot_counts(df, 'product_group', axs[0])
df.groupby('product_group')['product_group'].count().plot.pie(
    startangle=90, autopct="%.1f%%", ax=axs[1]);


df_nlp = df.copy()

# Create Features
df_nlp['length'] = df_nlp['text'].apply(len)
df_nlp['num_words'] = df_nlp['text'].str.split(' ').apply(len)
df_nlp['masked_XX'] = df_nlp['text'].str.split(' ').apply(lambda xx: sum(1 for w in xx if 'XX' in w.upper()))
df_nlp['caps_letter'] = df_nlp['text'].apply(lambda capsL: sum(1 for c in capsL if c.isupper()))
df_nlp['caps_word'] = df_nlp['text'].apply(lambda capsW : sum(1 for c in capsW.split() if c.isupper()))
df_nlp['exclaimations'] = df_nlp['text'].apply(lambda exclaim : exclaim.count('!'))
df_nlp['questions'] = df_nlp['text'].apply(lambda questions : questions.count('?'))
df_nlp['punct'] = df_nlp['text'].apply(lambda punct: sum(punct.count(p) for p in string.punctuation))
df_nlp['sp_char'] = df_nlp['text'].apply(lambda sp_char: sum(sp_char.count(s) for s in '@#$%^*()-'))
df_nlp['unique_words'] = df_nlp['text'].apply(lambda text: len(set(w for w in text.split())))



# Manipulate to find ratios
df_nlp['capsL_to_length'] = df_nlp['caps_letter'] / df_nlp['length']
df_nlp['capsW_to_length'] = df_nlp['caps_word'] / df_nlp['length']
df_nlp['capsW_to_words'] = df_nlp['caps_word'] / df_nlp['num_words']
df_nlp['unique_to_words'] = df_nlp['unique_words']/df_nlp['num_words']
df_nlp['masked_to_words'] = df_nlp['masked_XX']/df_nlp['num_words']


df_nlp.describe(exclude=['O'], percentiles=[0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.99])


df_nlp.describe(exclude=['O']).T


def explore_features(feature, threshold):
    
    df_feature = df_nlp.loc[df_nlp[feature] > threshold]
    bold_start = '\033[1m'
    bold_end = '\033[0m'
    print(bold_start+ "{} : ".format(feature) + bold_end + 
          "{} Percentage of records are with more than {} {} in train data and the count is {}".format(
              round(len(df_feature)/len(df_nlp)*100,2), threshold, feature, len(df_feature)))


# Using manually selected thresholds for CUSTOM FEATURES
for (i,j) in zip(['sp_char', 'exclaimations', 'questions', 'punct', 'masked_XX'],[15,5,5,75, 25]):
    explore_features(i, j)

sp_char : 4.8 Percentage of records are with more than 15 sp_char in train data and the count is 12890
exclaimations : 1.19 Percentage of records are with more than 5 exclaimations in train data and the count is 3194
questions : 0.56 Percentage of records are with more than 5 questions in train data and the count is 1495
punct : 7.22 Percentage of records are with more than 75 punct in train data and the count is 19370
masked_XX : 9.27 Percentage of records are with more than 25 masked_XX in train data and the count is 24872


# Using manually selected thresholds for BASIC FEATURES
for i,j in zip(['length', 'num_words', 'caps_word', 'unique_words'],[3500, 600, 70, 300]):
    explore_features(i, j)

length : 4.03 Percentage of records are with more than 3500 length in train data and the count is 10810
num_words : 4.86 Percentage of records are with more than 600 num_words in train data and the count is 13041
caps_word : 4.58 Percentage of records are with more than 70 caps_word in train data and the count is 12282
unique_words : 2.36 Percentage of records are with more than 300 unique_words in train data and the count is 6331


# Using manually selected thresholds for DERIVED RATIO FEATURES
for i,j in zip(['unique_to_words','capsL_to_length','capsW_to_words', 'masked_to_words'],
               [ 0.90, 0.20,0.25,0.15]):
    explore_features(i, j)

unique_to_words : 5.39 Percentage of records are with more than 0.9 unique_to_words in train data and the count is 14473
capsL_to_length : 5.29 Percentage of records are with more than 0.2 capsL_to_length in train data and the count is 14185
capsW_to_words : 5.07 Percentage of records are with more than 0.25 capsW_to_words in train data and the count is 13609
masked_to_words : 5.71 Percentage of records are with more than 0.15 masked_to_words in train data and the count is 15314


import warnings
warnings.filterwarnings('ignore')

def visualize_features(feature, threshold, X='product_group',):
    # Filter any extreme values
    _df = df_nlp[df_nlp[feature] <= threshold]
    
    f, axs = plt.subplots(1,2,figsize=(20,6))
    f.suptitle("Distribution of " + feature, weight='bold', size=16)
    
    my_palette = {"bank_service": "tab:blue", "credit_card": "tab:orange", "credit_reporting":"tab:green",
                  "debt_collection":"tab:red", "loan":"tab:purple" ,"money_transfers":"tab:brown" ,
                  "mortgage":"tab:pink"
                 }
    
    # Create Boxplot
    chart_ = sns.boxplot(x = X, y = feature, data = _df, ax=axs[0], palette=my_palette);
    chart_.set_xticklabels(chart_.get_xticklabels(),rotation=30)
    
    # Create Density plot
    for (i,j) in zip(df_nlp.product_group.unique(),my_palette.keys()):
        var = _df[_df.product_group == i][feature]
        sns.distplot(var, hist=False, label=i, ax=axs[1], fit_kws={"color":my_palette[j]})
        axs[1].legend()
    plt.show()


for (i,j) in zip(['sp_char', 'exclaimations', 'questions', 'punct', 'masked_XX'],[15,5,5,75,25]):
    visualize_features(i, j)


"""
sp_char :  No variation among different product groups - No information contained
exclaimations : too few to be valuable
questions : too few to be valuable
punct : distribution of punctuations varies across groups - will keep the counts, but remove in TEXT PROCESSING
masked_XX : 'Mortgage' has higest median and 'Debt_collection' has lowest median - varies across groups
            - will keep the counts, but remove in TEXT PROCESSING
"""


for i,j in zip(['length', 'num_words', 'caps_word', 'unique_words'],[3500, 600, 70, 300]):
    visualize_features(i, j)


"""
All of these features varies across groups. However, these features may have high correlation
Next step : Check correlation and discard features

"""


# dropping 'QUESTIONS' and 'EXCLAIMATIONS' as they are too few to be useful
# dropping 'SP_CHAR' as there is very little variation / information

correlation_matrix = df_nlp.drop(columns=['complaint_id','questions','exclaimations','sp_char']).corr()
mask = np.triu(np.ones_like(correlation_matrix, dtype=bool))

f, ax = plt.subplots(figsize=(20, 10))
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(correlation_matrix, mask=mask, cmap=cmap, center=0,
            square=True, linewidths=.5,
            vmin=-1, vmax=1, annot=True);


"""
1. "Length", "num_words" and "unique_words" - turned out to be most information in ditribution
2. "capsW_to_words" have very small correlation with "Length", "num_words" and "unique_words"
3. Same is true for "masked_to_words"
4. But "unique_to_words" shows higher correlation
5. Also, as expected "Length" is highly correlated with "num_words" and "unique_words"
    , as "num_words" is highly correlated with "unique_words" as well 
6. massed_xx and masked_to_words are also potential features
    
Conclusion:

Although these features look interesting, we will now move to NLP 
as the focus of the project is to analyze text data

"""


# for next steps we will only keep few columns
df_text = df.copy()


from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_selection import chi2

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split

import nltk
from nltk import sent_tokenize,word_tokenize
from nltk.corpus import stopwords
from tqdm import tqdm
from nltk.stem.porter import *
from nltk.stem import WordNetLemmatizer

import gensim


"""
First, we will explore the corpus before preprocessing or cleaning
This will give a broader idea of the underlying data

We will explore explore most and least frequent data

We already saw in EDA the punctuation or special characters are not useful, 
therefore will remove in text cleaning stage
"""


total_corpus = [i.split() for i in df_text['text'].values]


len(total_corpus)

268343


total_corpus_word = [w for sublist in total_corpus for w in sublist]
unique_corpus_word = list(set(total_corpus_word))
print("total words = ", len(total_corpus_word), "total unique words = ", len(unique_corpus_word))

total words =  52848489 total unique words =  285139


df_text['length'] = df_text['text'].str.split(' ').apply(len)
print('Mean number of words in document = ',df_text['length'].mean())

Mean number of words in document =  199.00576500970772


"""
Now we will look at most frequent words in corpus
"""


from sklearn.feature_extraction.text import CountVectorizer

def create_ngrams(text, ngram, top_n_words):
    text = text.apply(lambda x: np.str_(x))
    cv = CountVectorizer(ngram_range=(ngram, ngram)).fit(text)
    bag_of_words = cv.transform(text)
    sum_of_words = bag_of_words.sum(axis=0) 
    words_freq = [(w, sum_of_words[0, i]) for w, i in cv.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    
    return words_freq[:top_n_words], dict(words_freq[:top_n_words])


_, unigram = create_ngrams(df_text['text'],1,50)
_, bigram = create_ngrams(df_text['text'],2,50)
_, trigram = create_ngrams(df_text['text'],3,50)


fig, axs = plt.subplots(1,3,figsize = (25,10))
fig.suptitle("Most frequent 50 ngrams", weight='bold', size=20)
sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');


def most_freq_word(df, n=10, show_top_n=True):
    _, unigram = create_ngrams(df['text'],1,n)
    _, bigram = create_ngrams(df['text'],2,n)
    _, trigram = create_ngrams(df['text'],3,n)
    
    if show_top_n:
        fig, axs = plt.subplots(1,3,figsize = (25,10))
        fig.suptitle("Most frequent %d ngrams - " %n + i, weight='bold', size=20)
        sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
        sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
        sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');
    
    return unigram, bigram, trigram
    
for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    _,_,_ = most_freq_word(_df)

most_freq_monogram = {}
most_freq_bigram = {}
most_freq_trigram = {}

for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    m,b,t = most_freq_word(_df, n=1000, show_top_n=False)
    m_word = list(m.keys())
    b_word = list(b.keys())
    t_word = list(t.keys())
    most_freq_monogram[i] = m_word
    most_freq_bigram[i] = b_word
    most_freq_trigram[i] = t_word

most_freq_monogram = pd.DataFrame(most_freq_monogram)
most_freq_bigram = pd.DataFrame(most_freq_bigram)
most_freq_trigram = pd.DataFrame(most_freq_trigram)


most_freq_monogram.head()


most_freq_bigram.head()


most_freq_trigram.head()


"""
Next - will explore least frequent ngrams in corpus
"""


def create_ngrams2(text, ngram, top_n_words):
    text = text.apply(lambda x: np.str_(x))
    cv = CountVectorizer(ngram_range=(ngram, ngram)).fit(text)
    bag_of_words = cv.transform(text)
    sum_of_words = bag_of_words.sum(axis=0) 
    words_freq = [(w, sum_of_words[0, i]) for w, i in cv.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1])
    
    return words_freq[:top_n_words], dict(words_freq[:top_n_words])


_, unigram = create_ngrams2(df_text['text'],1,50)
_, bigram = create_ngrams2(df_text['text'],2,50)
_, trigram = create_ngrams2(df_text['text'],3,50)


fig, axs = plt.subplots(1,3,figsize = (25,10))
fig.suptitle("Least frequent 50 ngrams", weight='bold', size=20)
sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');


def least_freq_word(df, n=10, show_top_n=True):
    _, unigram = create_ngrams2(df['text'],1,n)
    _, bigram = create_ngrams2(df['text'],2,n)
    _, trigram = create_ngrams2(df['text'],3,n)
    
    if show_top_n:
        fig, axs = plt.subplots(1,3,figsize = (25,10))
        fig.suptitle("Least frequent %d ngrams - " %n + i, weight='bold', size=20)
        sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
        sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
        sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');
    
    return unigram, bigram, trigram
    
for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    _,_,_ = least_freq_word(_df)

l_freq_monogram = {}
l_freq_bigram = {}
l_freq_trigram = {}

for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    m,b,t = least_freq_word(_df, n=1000, show_top_n=False)
    m_word = list(m.keys())
    b_word = list(b.keys())
    t_word = list(t.keys())
    l_freq_monogram[i] = m_word
    l_freq_bigram[i] = b_word
    l_freq_trigram[i] = t_word

l_freq_monogram = pd.DataFrame(l_freq_monogram)
l_freq_bigram = pd.DataFrame(l_freq_bigram)
l_freq_trigram = pd.DataFrame(l_freq_trigram)


l_freq_monogram.head()


l_freq_bigram.head()


l_freq_trigram.head()


"""
At this stage we will clean the text by pre-processing

We observed the punctuations, including 'Question marks' and ' Exclaimations' are not useful in EDA

Also, the stop words are appearing everywhere and overcrowding the corpus without adding any valuable information

Next, we will apply lemmatization. Although Stemming could be applied, we chose Lemmatization in this step

The following function 'pre_preprocess_text' will help us just do all these steps

"""


def pre_preprocess_text(text, stemming=False, lemmatizing=True, sw=None, punct_list = None):
    ## clean (convert to lowercase and remove punctuations and characters and then strip)
    text = re.sub(r'[^\w\s]', '', str(text).lower().strip())
            
    ## Tokenize (convert from string to list)
    text_token = text.split()
    ## remove Stopwords
    if sw is not None:
        text_stopwordFree = [word for word in text_token if word not in sw]
        
    # Checking all words is alphabets or not - if not remove
    text_alphabets = [i for i in text_stopwordFree if i.isalpha()]
    
    # punctuations free
    if punct_list is not None:
        text_punctFree = [w for w in text_alphabets if w not in punct_list]
                
    ## Stemming (remove -ing, -ly, ...)
    if stemming == True:
        ps = nltk.stem.porter.PorterStemmer()
        text_clean = [ps.stem(word) for word in text_punctFree]
                
    ## Lemmatisation (convert the word into root word)
    if lemmatizing == True:
        lem = nltk.stem.wordnet.WordNetLemmatizer()
        text_clean = [lem.lemmatize(word) for word in text_punctFree]
            
    ## back to string from list
    text = " ".join(text_clean)
    
    return text


stopwords_list = nltk.corpus.stopwords.words("english")
puncts = [i for i in string.punctuation]

df_text['clean_text'] = df_text['text'].apply(lambda x: pre_preprocess_text(x, sw=stopwords_list, 
                                                                            punct_list = puncts))


"""
After text processing, we want to revisit most frequent words in corpus and in each product groups
"""


_, unigram = create_ngrams(df_text['clean_text'],1,50)
_, bigram = create_ngrams(df_text['clean_text'],2,50)
_, trigram = create_ngrams(df_text['clean_text'],3,50)


fig, axs = plt.subplots(1,3,figsize = (25,10))
fig.suptitle("Most frequent 50 ngrams", weight='bold', size=20)
sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');


def most_freq_word(df, n=10, show_top_n=True):
    _, unigram = create_ngrams(df['clean_text'],1,n)
    _, bigram = create_ngrams(df['clean_text'],2,n)
    _, trigram = create_ngrams(df['clean_text'],3,n)
    
    if show_top_n:
        fig, axs = plt.subplots(1,3,figsize = (25,10))
        fig.suptitle("Most frequent %d ngrams - " %n + i, weight='bold', size=20)
        sns.barplot(x=list(unigram.values()),y=list(unigram.keys()),ax=axs[0]); axs[0].set_title('Monogram')
        sns.barplot(x=list(bigram.values()),y=list(bigram.keys()),ax=axs[1]); axs[1].set_title('Bigram')
        sns.barplot(x=list(trigram.values()),y=list(trigram.keys()),ax=axs[2]); axs[2].set_title('Trigram');
    
    return unigram, bigram, trigram
    
for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    _,_,_ = most_freq_word(_df)

most_freq_monogram = {}
most_freq_bigram = {}
most_freq_trigram = {}

for i in df_text.product_group.unique():
    _df = df_text.loc[df_text.product_group == i]
    m,b,t = most_freq_word(_df, n=1000, show_top_n=False)
    m_word = list(m.keys())
    b_word = list(b.keys())
    t_word = list(t.keys())
    most_freq_monogram[i] = m_word
    most_freq_bigram[i] = b_word
    most_freq_trigram[i] = t_word

most_freq_monogram = pd.DataFrame(most_freq_monogram)
most_freq_bigram = pd.DataFrame(most_freq_bigram)
most_freq_trigram = pd.DataFrame(most_freq_trigram)


most_freq_monogram.head()


most_freq_bigram.head()


most_freq_trigram.head()


for i,j in zip(['masked_XX', 'masked_to_words'],
               [ 15,0.15]):
    explore_features(i, j)

masked_XX : 18.79 Percentage of records are with more than 15 masked_XX in train data and the count is 50432
masked_to_words : 5.71 Percentage of records are with more than 0.15 masked_to_words in train data and the count is 15314


for (i,j) in zip(['masked_XX','masked_to_words'],[15,0.15]):
    visualize_features(i, j)


"""
1. We can clearly see masking is overcrwoding the text data - but also saw there is variations in number 
    of masked words across different groups - Therefore, decided to keep them 

2. We used Bag of Words (BoW) to get most and least frequent words

3. Now, finally we will build TF-IDF - which will be used in the model development

"""


# PCA - using TF-IDF


from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

tfv = TfidfVectorizer(min_df=3,  max_features=1000, 
            strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
            ngram_range=(1, 1), use_idf=1,smooth_idf=1,sublinear_tf=1,
            stop_words = 'english')

text_pca = df_text['clean_text']
tfv.fit(text_pca)
text_pca_tfv =  tfv.transform(text_pca)

X = text_pca_tfv.todense()

reduced_data = PCA(n_components=2).fit_transform(X)

my_palette = {"bank_service": "tab:blue", "credit_card": "tab:orange", "credit_reporting":"tab:green",
                  "debt_collection":"tab:red", "loan":"tab:purple" ,"money_transfers":"tab:brown" ,
                  "mortgage":"tab:pink"
                 }

clss = df_text['product_group'].values.reshape(len(df_text),1)
X_pca = pd.DataFrame(np.hstack((reduced_data,clss)), columns = ['PCA1', 'PCA2','class'])

# this was with 1000 features
X_pca['color'] = X_pca['class'].map(my_palette)

plt.figure(figsize = (12,6))
sns.scatterplot(data=X_pca, x='PCA1', y='PCA2', hue='class');


X_pca.head()


"""
The PCA analysis shows the 2 dimensional projection of high dimension TF-IFD vectors

The PCA results suggests that it will be easier to idenfy few product groups, whereas in some
product groups, finding the correct classes may be difficult

Overall the plot is promosing and we can expect to generate satisfying results
"""


def run_TFIDF(corpus, custom_stopwords=True, vocabs = None):
    
        
    if custom_stopwords:
        sw = set(to_remove)
    else:
        sw = 'english'
        
    if vocabs == None:
        print ("Creating TF-IDF based on MonoGram and BiGram only due to computation limitations...")
        print ("Limiting maximum features to 10k in TF-IDF only due to computation limitations...")
        
        tfv = TfidfVectorizer(min_df=3,  max_features=10000, 
                              strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
                              ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1,
                              stop_words = sw)
        
    else:
        print ("Creating TF-IDF based on MonoGram and BiGram only due to computation limitations...")
        print ("Limiting maximum features to %d in TF-IDF only due to computation limitations..." %len(vocabs))
        v_list = vocabs
        tfv = TfidfVectorizer(min_df=3,  max_features=10000, 
                              strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
                              ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1,
                              stop_words = sw, vocabulary = v_list)
        
    

    print("fitting TF-IDF ...")
    tfv.fit(corpus)
    
    print("transforming TF-IDF ...")
    text_tfv =  tfv.transform(corpus)
    dic_tfv = tfv.vocabulary_
    
    return dic_tfv, tfv, text_tfv


def get_most_predictive_features(df, top_n, custom_sw=True, show_top_5=True):
    
    corpus = df['clean_text']
    dic_tfv, tfv, text_tfv = run_TFIDF(corpus, custom_sw)
    
    print('Converting from Sparse to Dense...')
    features = text_tfv.toarray()
    print('Convertion to Dense complete...')
    
    top_n_features = []
    
    for product in df_text.product_group.unique():
        features_chi2 = chi2(features, df_text["product_group"] == product)
        indices = np.argsort(features_chi2[0])
        feature_names = np.array(tfv.get_feature_names())[indices]
        unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
        bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
        top_n_features.append(unigrams[-top_n:])
        
        if show_top_5:
            print("n==> %s:" %(product))
            print("  * Most Correlated Unigrams are: %s" %(', '.join(unigrams[-5:])))
            print("  * Most Correlated Bigrams are: %s" %(', '.join(bigrams[-5:])))
            
    return dic_tfv, tfv, text_tfv, top_n_features


"""
Implemented Chi-Square test to identify most predictive co-efficients to classify the text data

We also printed out top 5 most useful 'unigrams' and 'bigrams' for each product groups

Finally, these also helped us reducing the number of most important features from 7000 to 2541

For next steps, these 2541 most informative unigrams will be used for model building
"""


dic_tfv, tfv, text_tfv, top_n_each = get_most_predictive_features(df_text, 1000, custom_sw=False)
                             
top_n_each_flat = [w for sublist in top_n_each for w in sublist]
top_n_uniques = list(set(top_n_each_flat))
print(len(top_n_each_flat), "==>",len(top_n_uniques))

Creating TF-IDF based on MonoGram and BiGram only due to computation limitations...
Limiting maximum features to 10k in TF-IDF only due to computation limitations...
fitting TF-IDF ...
transforming TF-IDF ...
Converting from Sparse to Dense...
Convertion to Dense complete...
n==> bank_service:
  * Most Correlated Unigrams are: branch, checking, bank, deposit, overdraft
  * Most Correlated Bigrams are: direct deposit, debit card, saving account, overdraft fee, checking account
n==> credit_card:
  * Most Correlated Unigrams are: amex, citi, express, reward, card
  * Most Correlated Bigrams are: card company, balance transfer, annual fee, american express, credit card
n==> credit_reporting:
  * Most Correlated Unigrams are: transunion, report, experian, inquiry, equifax
  * Most Correlated Bigrams are: xxxx equifax, equifax xxxx, credit file, hard inquiry, credit report
n==> debt_collection:
  * Most Correlated Unigrams are: collector, recovery, collect, collection, debt
  * Most Correlated Bigrams are: trying collect, debt collector, debt collection, collect debt, collection agency
n==> loan:
  * Most Correlated Unigrams are: school, repayment, student, loan, navient
  * Most Correlated Bigrams are: repayment plan, loan forgiveness, pay loan, private loan, student loan
n==> money_transfers:
  * Most Correlated Unigrams are: bitcoin, wire, paypal, western, coinbase
  * Most Correlated Bigrams are: sent money, money transfer, wire transfer, coinbase account, western union
n==> mortgage:
  * Most Correlated Unigrams are: property, foreclosure, escrow, modification, mortgage
  * Most Correlated Bigrams are: short sale, escrow account, mortgage company, mortgage payment, loan modification
7000 ==> 2541


dic_tfv, tfv, text_tfv, top_n_each = get_most_predictive_features(df_text, 1000, custom_sw=False)
                             
top_n_each_flat = [w for sublist in top_n_each for w in sublist]
top_n_uniques = list(set(top_n_each_flat))
print(len(top_n_each_flat), "==>",len(top_n_uniques))

Creating TF-IDF based on MonoGram and BiGram only due to computation limitations...
Limiting maximum features to 10k in TF-IDF only due to computation limitations...
fitting TF-IDF ...
transforming TF-IDF ...
Converting from Sparse to Dense...
Convertion to Dense complete...
n==> bank_service:
  * Most Correlated Unigrams are: branch, checking, bank, deposit, overdraft
  * Most Correlated Bigrams are: direct deposit, debit card, saving account, overdraft fee, checking account
n==> credit_card:
  * Most Correlated Unigrams are: amex, citi, express, reward, card
  * Most Correlated Bigrams are: card company, balance transfer, annual fee, american express, credit card
n==> credit_reporting:
  * Most Correlated Unigrams are: transunion, report, experian, inquiry, equifax
  * Most Correlated Bigrams are: xxxx equifax, equifax xxxx, credit file, hard inquiry, credit report
n==> debt_collection:
  * Most Correlated Unigrams are: collector, recovery, collect, collection, debt
  * Most Correlated Bigrams are: trying collect, debt collector, debt collection, collect debt, collection agency
n==> loan:
  * Most Correlated Unigrams are: school, repayment, student, loan, navient
  * Most Correlated Bigrams are: repayment plan, loan forgiveness, pay loan, private loan, student loan
n==> money_transfers:
  * Most Correlated Unigrams are: bitcoin, wire, paypal, western, coinbase
  * Most Correlated Bigrams are: sent money, money transfer, wire transfer, coinbase account, western union
n==> mortgage:
  * Most Correlated Unigrams are: property, foreclosure, escrow, modification, mortgage
  * Most Correlated Bigrams are: short sale, escrow account, mortgage company, mortgage payment, loan modification
7000 ==> 2541


# sample 100 words and see correlation heatmap
sns.heatmap(text_tfv.todense()[:,np.random.randint(0,1000,100)]==0, 
            vmin=0, vmax=1, cbar=False).set_title('Sparse Matrix Sample');


tfidf_df = pd.DataFrame(text_tfv.toarray(),columns=tfv.get_feature_names())


tfidf_df_sample.head()


"""
To begin the model building step - we will start with Multinomial Naive Bayes and Logistic Regression as baseline

Next we will move on to build better models

Also, we will use GridSearch and / or RandomSearch for hyperparameter tuning to find best models

Finally we will evaluate the models on the basis of F1 score and also look at the  AUC and ROC and PR plots
"""


from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC

models = [MultinomialNB()
          , LogisticRegression(random_state=43)
          , DecisionTreeClassifier()
#           ,LinearSVC()
         ]


corpus = df_text['clean_text']
dic_tfv, tfv, text_tfv_reduced = run_TFIDF(corpus,custom_stopwords=False, vocabs = top_n_uniques)
    
features_reduced = text_tfv_reduced.toarray()
print(features_reduced.shape)
product_groups_y = df_text['product_group'].values

# 5 Cross-validation
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))

entries = []
for model in models:
    model_name = model.__class__.__name__
    
    print("fitting %s model ... " %str(model_name))
    accuracies = cross_val_score(model, text_tfv_reduced, product_groups_y, scoring='accuracy', cv=CV)
    print("%s model has accruacy = " %str(model_name), accuracies)
    
    for fold_idx, accuracy in enumerate(accuracies):
        entries.append((model_name, fold_idx, accuracy))

cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'])

mean_accuracy = cv_df.groupby('model_name').accuracy.mean()
std_accuracy = cv_df.groupby('model_name').accuracy.std()

acc = pd.concat([mean_accuracy, std_accuracy], axis= 1, 
          ignore_index=True)
acc.columns = ['Mean Accuracy', 'Standard deviation']
acc

Creating TF-IDF based on MonoGram and BiGram only due to computation limitations...
Limiting maximum features to 2541 in TF-IDF only due to computation limitations...
fitting TF-IDF ...
transforming TF-IDF ...
(268343, 2541)
fitting MultinomialNB model ... 
MultinomialNB model has accruacy =  [0.81113865 0.81057966 0.80655499 0.81016621 0.81120966]
fitting LogisticRegression model ... 
LogisticRegression model has accruacy =  [0.84764016 0.8456092  0.8435596  0.84555042 0.84966833]
fitting DecisionTreeClassifier model ... 
DecisionTreeClassifier model has accruacy =  [0.76328979 0.76383014 0.76334569 0.75978237 0.76078855]


plt.figure(figsize=(8,5))
sns.boxplot(x='model_name', y='accuracy', 
            data=cv_df, 
            color='lightblue', 
            showmeans=True)
plt.title("MEAN ACCURACY (cv = 5)", size=14);


# !pip uninstall tensorflow 
# !pip install tensorflow


# import nltk
# import tensorflow as tf
# import keras


from sklearn.metrics import average_precision_score
from sklearn.metrics import precision_recall_curve
import itertools

def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """    
    Parameters
    ------------------------
    cm              : The confusion matrix to be plotted
    normalization   : A boolean value for normalizing the values in confusion matrix
    title           : Title of the plot
    cmap            : The color map to be used
    
    Returns
    -------------------------
    Plots the confusion matrix.
    
    PS-This function in called in do_classify.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

###############################################################################################    

def make_pr(name, clf, ytest, xtest, ax=None):
    
    """
    Parameters
    ------------------------
    name  : Name of the classifier in strings
    clf   : The tuned classifier
    ytest : The true responses in the test set
    xtest : Features of the test set
    ax    : The number of PR curves to print
    
    
    Returns
    -------------------------
    ax: Plots the PR curve along with AP value.
    
    PS: Code taken from http://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html
    """
   
    average_precision = average_precision_score(ytest, clf.predict_proba(xtest)[:,1])
    
    precision, recall, _ = precision_recall_curve(y_test, clf.predict_proba(xtest)[:,1])

    plt.step(recall, precision, color='b', alpha=0.2,
         where='post')
    
    plt.fill_between(recall, precision, step='post', alpha=0.2,
                 color='b')

    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.ylim([0.0, 1.05])
    plt.xlim([0.0, 1.0])
    plt.title('Precision-Recall curve: AP={0:0.2f}'.format(
          average_precision))


from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import f1_score, make_scorer

def cv_optimize(clf, parameters, X, y, n_jobs=-1, n_folds=5):
    
    """ 
    Parameters
    ------------------------
    clf          : The classifier
    parameters   : A dictionary of parameters on which grid search is to be performed
    X            : Dataframe of features
    y            : A Dataframe of responses/target
    n_jobs       : Parameter for number of cores to engage when performing cv
    n_folds      : The number of cv folds
    
    
    Returns
    -------------------------
    best : The best hyperparameters for the classifier
    """
    
    skf = StratifiedKFold(n_splits=n_folds, shuffle = True, random_state = 999)
    
    gs = GridSearchCV(clf, param_grid=parameters, cv=skf.split(X,y), n_jobs=n_jobs, 
                          scoring=f1)
    
    gs.fit(X, y)
    
    print ("BEST", gs.best_params_, gs.best_score_, gs.cv_results_)
    
    best = gs.best_estimator_
    
    return best

#######################################################################################


def rv_optimize(clf, parameters, X, y, n_jobs=-1, n_folds=5):
    
    """ 
    Parameters
    ------------------------
    clf          : The classifier
    parameters   : A dictionary of parameters on which grid search is to be performed
    X            : Dataframe of features
    y            : A Dataframe of responses/target
    n_jobs       : Parameter for number of cores to engage when performing cv
    n_folds      : The number of cv folds
    
    
    Returns
    -------------------------
    best : The best hyperparameters for the classifier
    """
    
    skf = StratifiedKFold(n_splits=n_folds, shuffle = True, random_state = 999)
    
    rs = RandomizedSearchCV(clf, param_distributions=parameters, cv=skf.split(X,y), n_jobs=n_jobs, 
                          scoring=f1)
    
    rs.fit(X, y)
    
    print ("BEST", gs.best_params_, gs.best_score_, gs.cv_results_)
    
    best = gs.best_estimator_
    
    return best

#######################################################################################

def do_classify(clf, parameters, Xtrain,Xtest,ytrain,ytest, n_folds=5, 
                n_jobs=-1):
    """
    Parameters
    ------------------------
    clf          : The classifier
    parameters   : A dictionary of parameters on which grid search is to be performed
    Xtrain       : Train Dataframe of features
    Xtest        : Test Dataframe of features
    ytrain       : Train Dataframe of responses/target
    ytest        : Test Dataframe of responses/target
    n_jobs       : Parameter for number of cores to engage when performing cv
    n_folds      : The number of cv folds
    
    
    Returns
    -------------------------
    clf          : The classifier with optimized hyperparameters
    Xtrain       : The Train Dataframe of features
    Xtest        : The Test Dataframe of features
    ytrain       : The Train Dataframe of responses/target
    ytest        : The Test Dataframe of responses/target
    
    Prints the training and test accuracy.
    Plots the confusion matrix.
    """
    #performing grid search to find best hyperparameters
    
    f1 = make_scorer(f1_score,average='weighted')

    if parameters:
        clf = cv_optimize(clf, parameters, Xtrain, ytrain, n_jobs=n_jobs, n_folds=n_folds)
    
    # Fitting the tuned classifier
    clf=clf.fit(Xtrain, ytrain)
    
    # Calculating the training accuracy
    training_accuracy = clf.score(Xtrain, ytrain)
    
    # Calculating the test accuracy
    test_accuracy = clf.score(Xtest, ytest)
    
        
    print ("############# based on standard predict ################")
    print ("Accuracy on training data: %0.2f" % (training_accuracy))
    print ("Accuracy on test data:     %0.2f" % (test_accuracy))
    
    #Calculating the confusion matrix
    cnf_matrix = confusion_matrix(ytest, clf.predict(Xtest))
    np.set_printoptions(precision=3)
    
    plt.figure()
    
    #Plotting the confusion matrix
    plot_confusion_matrix(cnf_matrix, classes=[0,1],
                      title='Confusion matrix, without normalization')
    plt.show()
    print ("########################################################")
    
    return clf, Xtrain, ytrain, Xtest, ytest



def tune_classifier(clf, parameters, Xtrain,Xtest,ytrain,ytest, n_folds=5, 
                n_jobs=-1, tuning='GridSearch'):
    """
    Parameters
    ------------------------
    clf          : The classifier
    parameters   : A dictionary of parameters on which grid search is to be performed
    Xtrain       : Train Dataframe of features
    Xtest        : Test Dataframe of features
    ytrain       : Train Dataframe of responses/target
    ytest        : Test Dataframe of responses/target
    n_jobs       : Parameter for number of cores to engage when performing cv
    n_folds      : The number of cv folds
    
    
    Returns
    -------------------------
    clf          : The classifier with optimized hyperparameters
    Xtrain       : The Train Dataframe of features
    Xtest        : The Test Dataframe of features
    ytrain       : The Train Dataframe of responses/target
    ytest        : The Test Dataframe of responses/target
    
    Prints the training and test accuracy.
    Plots the confusion matrix.
    """
    #performing grid search to find best hyperparameters
    
    f1 = make_scorer(f1_score,average='weighted')

    if parameters:
        if tuning=='GridSearch':
            clf = cv_optimize(clf, parameters, Xtrain, ytrain, n_jobs=n_jobs, n_folds=n_folds)
        elif tuning=='RandomSearch':
            clf = rv_optimize(clf, parameters, Xtrain, ytrain, n_jobs=n_jobs, n_folds=n_folds)
    
    # Fitting the tuned classifier
    clf=clf.fit(Xtrain, ytrain)
    
    # Calculating the training accuracy
    training_accuracy = clf.score(Xtrain, ytrain)
    
    # Calculating the test accuracy
    test_accuracy = clf.score(Xtest, ytest)
    
        
    print ("############# based on standard predict ################")
    print ("Accuracy on training data: %0.2f" % (training_accuracy))
    print ("Accuracy on test data:     %0.2f" % (test_accuracy))
    
    #Calculating the confusion matrix
    cnf_matrix = confusion_matrix(ytest, clf.predict(Xtest))
    np.set_printoptions(precision=3)
    
    plt.figure()
    
    #Plotting the confusion matrix
    plot_confusion_matrix(cnf_matrix, classes=[0,1],
                      title='Confusion matrix, without normalization')
    plt.show()
    print ("########################################################")
    
    return clf, Xtrain, ytrain, Xtest, ytest


vectorizer = TfidfVectorizer(min_df=3,  max_features=10000, 
                             strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
                             ngram_range=(1, 1), use_idf=1,smooth_idf=1,sublinear_tf=1,
                             stop_words = 'english', vocabulary = top_n_uniques)


corpus = df_text['clean_text']
X = vectorizer.fit_transform(corpus)
Y = df_text['product_group'].values
x_train, x_valid, y_train, y_valid = train_test_split(X, Y, random_state=43, test_size=0.2)


#Creating a f1 scoring function
f1 = make_scorer(f1_score,average='weighted')


from sklearn.naive_bayes import MultinomialNB

parameters = None
tune_classifier(MultinomialNB(),parameters, x_train, x_valid ,y_train,y_valid,n_folds=5, 
                n_jobs=-1)

############# based on standard predict ################
Accuracy on training data: 0.77
Accuracy on test data:     0.76
Confusion matrix, without normalization
[[ 1687   703   865   341    31     0   437]
 [   47  3377  1881   529    80     0   155]
 [    6    80 14923   728   131     0   182]
 [    9    56  2218  9488   164     0   192]
 [   16    84  1231   978  3664     0   334]
 [  176   238   226   169     7    39    91]
 [   11     9   474   179    10     0  7423]]

########################################################

(MultinomialNB(),
 <214674x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 12563571 stored elements in Compressed Sparse Row format>,
 array(['mortgage', 'mortgage', 'debt_collection', ..., 'credit_card',
        'mortgage', 'debt_collection'], dtype=object),
 <53669x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 3161710 stored elements in Compressed Sparse Row format>,
 array(['loan', 'bank_service', 'mortgage', ..., 'credit_reporting',
        'debt_collection', 'mortgage'], dtype=object))


from sklearn.linear_model import LogisticRegression

parameters = {"C": [0.01, 0.1, 1, 10, 100, 1000],
             "penalty":['l2','l1']}
tune_classifier(LogisticRegression(),parameters, x_train, x_valid ,y_train,y_valid,n_folds=5, 
                n_jobs=-1)

BEST {'C': 1, 'penalty': 'l2'} 0.8444761056143738 {'mean_fit_time': array([1.580e+02, 3.210e-01, 3.711e+02, 1.423e+00, 6.196e+02, 7.423e-01,
       2.930e+02, 1.535e+00, 1.920e+02, 3.293e+00, 1.136e+02, 1.199e+00]), 'std_fit_time': array([4.996e+00, 1.951e-01, 2.034e+02, 8.383e-01, 1.092e+01, 3.772e-01,
       1.516e+02, 1.530e+00, 2.953e+01, 1.483e+00, 4.913e+00, 4.364e-01]), 'mean_score_time': array([0.881, 0.   , 1.158, 0.   , 1.199, 0.   , 1.509, 0.   , 1.134,
       0.   , 0.589, 0.   ]), 'std_score_time': array([0.167, 0.   , 0.149, 0.   , 0.11 , 0.   , 0.355, 0.   , 0.361,
       0.   , 0.078, 0.   ]), 'param_C': masked_array(data=[0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100, 100, 1000,
                   1000],
             mask=[False, False, False, False, False, False, False, False,
                   False, False, False, False],
       fill_value='?',
            dtype=object), 'param_penalty': masked_array(data=['l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2',
                   'l1', 'l2', 'l1'],
             mask=[False, False, False, False, False, False, False, False,
                   False, False, False, False],
       fill_value='?',
            dtype=object), 'params': [{'C': 0.01, 'penalty': 'l2'}, {'C': 0.01, 'penalty': 'l1'}, {'C': 0.1, 'penalty': 'l2'}, {'C': 0.1, 'penalty': 'l1'}, {'C': 1, 'penalty': 'l2'}, {'C': 1, 'penalty': 'l1'}, {'C': 10, 'penalty': 'l2'}, {'C': 10, 'penalty': 'l1'}, {'C': 100, 'penalty': 'l2'}, {'C': 100, 'penalty': 'l1'}, {'C': 1000, 'penalty': 'l2'}, {'C': 1000, 'penalty': 'l1'}], 'split0_test_score': array([0.786,   nan, 0.835,   nan, 0.846,   nan, 0.842,   nan, 0.842,
         nan, 0.839,   nan]), 'split1_test_score': array([0.784,   nan, 0.831,   nan, 0.843,   nan, 0.841,   nan, 0.839,
         nan, 0.839,   nan]), 'split2_test_score': array([0.787,   nan, 0.834,   nan, 0.844,   nan, 0.841,   nan, 0.838,
         nan, 0.839,   nan]), 'split3_test_score': array([0.786,   nan, 0.832,   nan, 0.844,   nan, 0.845,   nan, 0.842,
         nan, 0.841,   nan]), 'split4_test_score': array([0.785,   nan, 0.833,   nan, 0.844,   nan, 0.84 ,   nan, 0.839,
         nan, 0.839,   nan]), 'mean_test_score': array([0.786,   nan, 0.833,   nan, 0.844,   nan, 0.842,   nan, 0.84 ,
         nan, 0.839,   nan]), 'std_test_score': array([0.001,   nan, 0.001,   nan, 0.001,   nan, 0.002,   nan, 0.001,
         nan, 0.001,   nan]), 'rank_test_score': array([ 6,  7,  5,  8,  1,  9,  2, 10,  3, 11,  4, 12], dtype=int32)}
############# based on standard predict ################
Accuracy on training data: 0.86
Accuracy on test data:     0.85
Confusion matrix, without normalization
[[ 3365   278   108    94    44    86    89]
 [  294  4808   496   328    80    30    33]
 [   71   370 13969  1024   399     1   216]
 [   71   210  1090 10175   411     8   162]
 [   91   128   468   565  4932     4   119]
 [  169    46    16    21    16   668    10]
 [   71    33   223   170   103     0  7506]]

########################################################

(LogisticRegression(C=1),
 <214674x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 12563571 stored elements in Compressed Sparse Row format>,
 array(['mortgage', 'mortgage', 'debt_collection', ..., 'credit_card',
        'mortgage', 'debt_collection'], dtype=object),
 <53669x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 3161710 stored elements in Compressed Sparse Row format>,
 array(['loan', 'bank_service', 'mortgage', ..., 'credit_reporting',
        'debt_collection', 'mortgage'], dtype=object))


from sklearn.ensemble import RandomForestClassifier

parameters = {"n_estimators": [50,100,150,200], "max_depth":range(2,20,1)}
do_classify(RandomForestClassifier(),parameters, x_train, x_valid ,y_train,y_valid,n_folds=5, 
                n_jobs=-1)

BEST {'max_depth': 19, 'n_estimators': 200} 0.5530530118331888 {'mean_fit_time': array([   6.78 ,   11.772,   17.439,   23.273,    9.266,   16.855,
         26.36 ,   36.162,   12.443,   24.289,   41.292,   51.646,
         16.408,   31.738,   43.858,   59.268,   17.811,   36.306,
         49.478,   68.451,   20.375,   41.313,   61.258,   81.742,
         24.377,   48.592,   69.413,   94.223,   25.847,   51.222,
         77.689,  105.218,   31.363,   57.833,   91.828,  115.377,
         33.997,  623.312, 1020.861,  122.175,   34.873,   66.463,
        100.381,  140.171,   38.544,   72.452,  116.578,  162.438,
         47.231,   92.324,  127.117,  173.423,   46.025,   95.147,
        135.644,  194.358,   51.419,  103.588,  161.516,  240.824,
         61.408,  136.774,  202.583,  273.74 ,   69.707,  141.759,
        208.645,  275.704,   76.065,  153.447,  227.8  ,  249.694]), 'std_fit_time': array([2.632e-01, 1.699e-01, 8.264e-01, 1.182e+00, 8.289e-01, 4.710e-01,
       1.547e+00, 3.207e+00, 9.131e-01, 1.676e+00, 3.347e+00, 3.451e+00,
       1.049e+00, 2.542e+00, 1.817e+00, 5.442e+00, 6.012e-01, 2.883e+00,
       1.746e+00, 6.214e+00, 6.875e-01, 2.494e+00, 4.555e+00, 4.970e+00,
       2.530e+00, 3.249e+00, 5.613e+00, 8.089e+00, 6.448e-01, 1.858e+00,
       7.400e+00, 6.887e+00, 4.265e+00, 2.256e+00, 6.509e+00, 9.534e+00,
       1.902e+00, 4.592e+02, 7.109e+00, 7.022e+00, 3.079e+00, 5.223e+00,
       6.020e+00, 1.178e+01, 2.840e+00, 2.972e+00, 9.430e+00, 1.581e+01,
       1.940e+00, 6.210e+00, 4.612e+00, 1.438e+01, 1.792e+00, 7.259e+00,
       6.281e+00, 1.387e+01, 1.686e+00, 9.302e+00, 9.394e+00, 2.215e+01,
       2.408e+00, 4.460e+00, 1.178e+01, 1.955e+01, 1.407e+00, 4.671e+00,
       1.798e+01, 1.968e+01, 5.177e+00, 7.842e+00, 1.335e+01, 4.364e+01]), 'mean_score_time': array([1.052, 1.436, 2.004, 2.52 , 1.134, 1.639, 2.211, 2.983, 1.171,
       2.274, 2.613, 3.03 , 1.327, 1.945, 2.37 , 3.044, 1.323, 1.723,
       2.382, 2.974, 1.241, 1.846, 2.343, 2.978, 1.262, 1.824, 2.367,
       2.974, 1.213, 1.851, 2.362, 3.639, 1.392, 1.983, 2.398, 3.144,
       1.26 , 1.9  , 2.23 , 2.742, 1.217, 1.777, 2.362, 3.064, 1.302,
       1.897, 2.646, 3.492, 1.639, 2.118, 2.667, 3.299, 1.392, 2.055,
       2.766, 3.433, 1.406, 2.434, 3.166, 4.18 , 1.683, 2.584, 3.59 ,
       4.519, 1.751, 2.894, 3.477, 4.775, 1.882, 2.727, 3.172, 2.565]), 'std_score_time': array([0.034, 0.059, 0.142, 0.086, 0.128, 0.122, 0.139, 0.214, 0.083,
       0.214, 0.283, 0.211, 0.034, 0.08 , 0.062, 0.121, 0.181, 0.066,
       0.136, 0.222, 0.072, 0.058, 0.104, 0.22 , 0.083, 0.093, 0.154,
       0.084, 0.074, 0.121, 0.068, 0.931, 0.117, 0.11 , 0.071, 0.189,
       0.04 , 0.175, 0.199, 0.093, 0.083, 0.052, 0.076, 0.117, 0.067,
       0.108, 0.221, 0.249, 0.248, 0.153, 0.126, 0.058, 0.055, 0.064,
       0.045, 0.09 , 0.051, 0.491, 0.492, 0.323, 0.13 , 0.263, 0.378,
       0.694, 0.252, 0.551, 0.437, 0.713, 0.113, 0.164, 0.253, 0.592]), 'param_max_depth': masked_array(data=[2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6,
                   6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10,
                   10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14,
                   14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17,
                   17, 18, 18, 18, 18, 19, 19, 19, 19],
             mask=[False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False],
       fill_value='?',
            dtype=object), 'param_n_estimators': masked_array(data=[50, 100, 150, 200, 50, 100, 150, 200, 50, 100, 150,
                   200, 50, 100, 150, 200, 50, 100, 150, 200, 50, 100,
                   150, 200, 50, 100, 150, 200, 50, 100, 150, 200, 50,
                   100, 150, 200, 50, 100, 150, 200, 50, 100, 150, 200,
                   50, 100, 150, 200, 50, 100, 150, 200, 50, 100, 150,
                   200, 50, 100, 150, 200, 50, 100, 150, 200, 50, 100,
                   150, 200, 50, 100, 150, 200],
             mask=[False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False,
                   False, False, False, False, False, False, False, False],
       fill_value='?',
            dtype=object), 'params': [{'max_depth': 2, 'n_estimators': 50}, {'max_depth': 2, 'n_estimators': 100}, {'max_depth': 2, 'n_estimators': 150}, {'max_depth': 2, 'n_estimators': 200}, {'max_depth': 3, 'n_estimators': 50}, {'max_depth': 3, 'n_estimators': 100}, {'max_depth': 3, 'n_estimators': 150}, {'max_depth': 3, 'n_estimators': 200}, {'max_depth': 4, 'n_estimators': 50}, {'max_depth': 4, 'n_estimators': 100}, {'max_depth': 4, 'n_estimators': 150}, {'max_depth': 4, 'n_estimators': 200}, {'max_depth': 5, 'n_estimators': 50}, {'max_depth': 5, 'n_estimators': 100}, {'max_depth': 5, 'n_estimators': 150}, {'max_depth': 5, 'n_estimators': 200}, {'max_depth': 6, 'n_estimators': 50}, {'max_depth': 6, 'n_estimators': 100}, {'max_depth': 6, 'n_estimators': 150}, {'max_depth': 6, 'n_estimators': 200}, {'max_depth': 7, 'n_estimators': 50}, {'max_depth': 7, 'n_estimators': 100}, {'max_depth': 7, 'n_estimators': 150}, {'max_depth': 7, 'n_estimators': 200}, {'max_depth': 8, 'n_estimators': 50}, {'max_depth': 8, 'n_estimators': 100}, {'max_depth': 8, 'n_estimators': 150}, {'max_depth': 8, 'n_estimators': 200}, {'max_depth': 9, 'n_estimators': 50}, {'max_depth': 9, 'n_estimators': 100}, {'max_depth': 9, 'n_estimators': 150}, {'max_depth': 9, 'n_estimators': 200}, {'max_depth': 10, 'n_estimators': 50}, {'max_depth': 10, 'n_estimators': 100}, {'max_depth': 10, 'n_estimators': 150}, {'max_depth': 10, 'n_estimators': 200}, {'max_depth': 11, 'n_estimators': 50}, {'max_depth': 11, 'n_estimators': 100}, {'max_depth': 11, 'n_estimators': 150}, {'max_depth': 11, 'n_estimators': 200}, {'max_depth': 12, 'n_estimators': 50}, {'max_depth': 12, 'n_estimators': 100}, {'max_depth': 12, 'n_estimators': 150}, {'max_depth': 12, 'n_estimators': 200}, {'max_depth': 13, 'n_estimators': 50}, {'max_depth': 13, 'n_estimators': 100}, {'max_depth': 13, 'n_estimators': 150}, {'max_depth': 13, 'n_estimators': 200}, {'max_depth': 14, 'n_estimators': 50}, {'max_depth': 14, 'n_estimators': 100}, {'max_depth': 14, 'n_estimators': 150}, {'max_depth': 14, 'n_estimators': 200}, {'max_depth': 15, 'n_estimators': 50}, {'max_depth': 15, 'n_estimators': 100}, {'max_depth': 15, 'n_estimators': 150}, {'max_depth': 15, 'n_estimators': 200}, {'max_depth': 16, 'n_estimators': 50}, {'max_depth': 16, 'n_estimators': 100}, {'max_depth': 16, 'n_estimators': 150}, {'max_depth': 16, 'n_estimators': 200}, {'max_depth': 17, 'n_estimators': 50}, {'max_depth': 17, 'n_estimators': 100}, {'max_depth': 17, 'n_estimators': 150}, {'max_depth': 17, 'n_estimators': 200}, {'max_depth': 18, 'n_estimators': 50}, {'max_depth': 18, 'n_estimators': 100}, {'max_depth': 18, 'n_estimators': 150}, {'max_depth': 18, 'n_estimators': 200}, {'max_depth': 19, 'n_estimators': 50}, {'max_depth': 19, 'n_estimators': 100}, {'max_depth': 19, 'n_estimators': 150}, {'max_depth': 19, 'n_estimators': 200}], 'split0_test_score': array([0.142, 0.142, 0.141, 0.142, 0.15 , 0.147, 0.146, 0.148, 0.157,
       0.165, 0.161, 0.156, 0.19 , 0.18 , 0.193, 0.182, 0.219, 0.223,
       0.206, 0.214, 0.237, 0.241, 0.24 , 0.248, 0.267, 0.26 , 0.269,
       0.295, 0.315, 0.322, 0.306, 0.318, 0.325, 0.36 , 0.335, 0.351,
       0.404, 0.36 , 0.362, 0.376, 0.413, 0.393, 0.41 , 0.391, 0.439,
       0.446, 0.426, 0.433, 0.453, 0.449, 0.447, 0.447, 0.461, 0.469,
       0.47 , 0.468, 0.488, 0.496, 0.481, 0.496, 0.51 , 0.524, 0.511,
       0.519, 0.536, 0.521, 0.535, 0.535, 0.549, 0.557, 0.546, 0.551]), 'split1_test_score': array([0.142, 0.142, 0.142, 0.142, 0.149, 0.145, 0.145, 0.145, 0.17 ,
       0.169, 0.156, 0.164, 0.195, 0.189, 0.177, 0.186, 0.213, 0.213,
       0.206, 0.21 , 0.247, 0.264, 0.247, 0.256, 0.29 , 0.279, 0.272,
       0.272, 0.328, 0.308, 0.313, 0.31 , 0.306, 0.365, 0.359, 0.343,
       0.372, 0.358, 0.376, 0.367, 0.419, 0.389, 0.382, 0.389, 0.448,
       0.406, 0.421, 0.427, 0.454, 0.455, 0.439, 0.452, 0.483, 0.467,
       0.477, 0.478, 0.48 , 0.493, 0.486, 0.498, 0.502, 0.509, 0.506,
       0.503, 0.527, 0.544, 0.534, 0.53 , 0.527, 0.563, 0.54 , 0.549]), 'split2_test_score': array([0.142, 0.142, 0.143, 0.141, 0.145, 0.144, 0.144, 0.144, 0.193,
       0.17 , 0.154, 0.164, 0.207, 0.178, 0.195, 0.178, 0.219, 0.212,
       0.231, 0.204, 0.25 , 0.24 , 0.24 , 0.243, 0.317, 0.296, 0.301,
       0.264, 0.277, 0.312, 0.291, 0.307, 0.366, 0.325, 0.325, 0.321,
       0.357, 0.399, 0.373, 0.369, 0.378, 0.406, 0.396, 0.38 , 0.436,
       0.433, 0.429, 0.416, 0.463, 0.45 , 0.46 , 0.439, 0.473, 0.477,
       0.473, 0.469, 0.493, 0.497, 0.491, 0.501, 0.504, 0.519, 0.51 ,
       0.523, 0.521, 0.518, 0.527, 0.53 , 0.542, 0.549, 0.555, 0.559]), 'split3_test_score': array([0.142, 0.142, 0.142, 0.142, 0.146, 0.15 , 0.143, 0.146, 0.172,
       0.156, 0.156, 0.156, 0.196, 0.193, 0.185, 0.192, 0.229, 0.224,
       0.217, 0.214, 0.232, 0.239, 0.24 , 0.264, 0.276, 0.265, 0.272,
       0.274, 0.288, 0.3  , 0.299, 0.326, 0.358, 0.351, 0.334, 0.323,
       0.362, 0.35 , 0.363, 0.374, 0.396, 0.386, 0.378, 0.408, 0.426,
       0.434, 0.423, 0.416, 0.456, 0.449, 0.462, 0.434, 0.46 , 0.467,
       0.472, 0.46 , 0.488, 0.496, 0.486, 0.506, 0.513, 0.499, 0.515,
       0.507, 0.537, 0.526, 0.524, 0.536, 0.561, 0.538, 0.54 , 0.554]), 'split4_test_score': array([0.142, 0.142, 0.142, 0.142, 0.159, 0.149, 0.144, 0.146, 0.167,
       0.161, 0.163, 0.163, 0.17 , 0.196, 0.169, 0.174, 0.237, 0.262,
       0.219, 0.215, 0.243, 0.253, 0.255, 0.237, 0.302, 0.277, 0.281,
       0.267, 0.308, 0.311, 0.29 , 0.302, 0.363, 0.347, 0.353, 0.358,
       0.389, 0.399, 0.38 , 0.366, 0.376, 0.409, 0.395, 0.418, 0.42 ,
       0.421, 0.419, 0.427, 0.441, 0.461, 0.443, 0.439, 0.458, 0.474,
       0.493, 0.463, 0.477, 0.498, 0.503, 0.501, 0.5  , 0.514, 0.518,
       0.501, 0.51 , 0.552, 0.548, 0.524, 0.539, 0.549, 0.567, 0.552]), 'mean_test_score': array([0.142, 0.142, 0.142, 0.142, 0.15 , 0.147, 0.145, 0.146, 0.172,
       0.164, 0.158, 0.161, 0.192, 0.187, 0.184, 0.182, 0.224, 0.227,
       0.215, 0.211, 0.242, 0.247, 0.244, 0.25 , 0.291, 0.276, 0.279,
       0.274, 0.303, 0.311, 0.3  , 0.312, 0.344, 0.349, 0.341, 0.339,
       0.377, 0.373, 0.371, 0.37 , 0.396, 0.397, 0.392, 0.397, 0.434,
       0.428, 0.423, 0.424, 0.454, 0.453, 0.45 , 0.442, 0.467, 0.471,
       0.477, 0.468, 0.485, 0.496, 0.489, 0.5  , 0.506, 0.513, 0.512,
       0.51 , 0.526, 0.532, 0.534, 0.531, 0.543, 0.551, 0.55 , 0.553]), 'std_test_score': array([0.   , 0.   , 0.   , 0.   , 0.005, 0.002, 0.001, 0.001, 0.012,
       0.005, 0.003, 0.004, 0.012, 0.007, 0.01 , 0.006, 0.009, 0.018,
       0.009, 0.004, 0.007, 0.01 , 0.006, 0.009, 0.018, 0.012, 0.012,
       0.011, 0.018, 0.007, 0.009, 0.009, 0.024, 0.014, 0.013, 0.015,
       0.017, 0.022, 0.007, 0.004, 0.017, 0.009, 0.012, 0.014, 0.01 ,
       0.013, 0.004, 0.007, 0.007, 0.005, 0.009, 0.006, 0.01 , 0.004,
       0.008, 0.006, 0.006, 0.002, 0.007, 0.003, 0.005, 0.008, 0.004,
       0.009, 0.01 , 0.014, 0.008, 0.004, 0.011, 0.008, 0.01 , 0.003]), 'rank_test_score': array([70, 71, 69, 72, 65, 66, 68, 67, 61, 62, 64, 63, 57, 58, 59, 60, 54,
       53, 55, 56, 52, 50, 51, 49, 45, 47, 46, 48, 43, 42, 44, 41, 38, 37,
       39, 40, 33, 34, 35, 36, 31, 30, 32, 29, 25, 26, 28, 27, 21, 22, 23,
       24, 20, 18, 17, 19, 16, 14, 15, 13, 12,  9, 10, 11,  8,  6,  5,  7,
        4,  2,  3,  1], dtype=int32)}
############# based on standard predict ################
Accuracy on training data: 0.61
Accuracy on test data:     0.58
Confusion matrix, without normalization
[[  975    58  2742   108    10     0   171]
 [   36  1101  4534   255    11     0   132]
 [    4     7 15745   161     9     0   124]
 [    3    15  5960  5928    43     0   178]
 [   14    14  3586   611  1654     0   428]
 [   44    11   807    36     3     0    45]
 [    4     0  2279    94     0     0  5729]]

########################################################

(RandomForestClassifier(max_depth=19, n_estimators=200),
 <214674x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 12563571 stored elements in Compressed Sparse Row format>,
 array(['mortgage', 'mortgage', 'debt_collection', ..., 'credit_card',
        'mortgage', 'debt_collection'], dtype=object),
 <53669x122962 sparse matrix of type '<class 'numpy.float64'>'
 	with 3161710 stored elements in Compressed Sparse Row format>,
 array(['loan', 'bank_service', 'mortgage', ..., 'credit_reporting',
        'debt_collection', 'mortgage'], dtype=object))


parameters = None

tune_classifier(GradientBoostingClassifier(),parameters, x_train, x_valid ,y_train,y_valid,n_folds=5, 
                n_jobs=-1)

############# based on standard predict ################
Accuracy on training data: 0.82
Accuracy on test data:     0.81
Confusion matrix, without normalization
[[ 3125   377   143   158    71   109    81]
 [  267  4547   665   477    59    29    25]
 [   69   362 13548  1296   488     3   284]
 [   84   275  1282  9852   419     8   207]
 [   95    86   563   694  4767     6    96]
 [  163    45    29    54    10   632    13]
 [   89    28   249   232   295     1  7212]]

########################################################

(GradientBoostingClassifier(),
 <214674x2541 sparse matrix of type '<class 'numpy.float64'>'
 	with 9090335 stored elements in Compressed Sparse Row format>,
 array(['mortgage', 'mortgage', 'debt_collection', ..., 'credit_card',
        'mortgage', 'debt_collection'], dtype=object),
 <53669x2541 sparse matrix of type '<class 'numpy.float64'>'
 	with 2288391 stored elements in Compressed Sparse Row format>,
 array(['loan', 'bank_service', 'mortgage', ..., 'credit_reporting',
        'debt_collection', 'mortgage'], dtype=object))


parameters = {'max_depth' : [3, 5, 8],
              'n_estimators' : [50, 100, 200],
              'learning_rate':[0.01,0.05,0.1]}

tune_classifier(GradientBoostingClassifier(),parameters, x_train, x_valid ,y_train,y_valid,n_folds=5, 
                n_jobs=-1,tuning='RandomSearch')


from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, classification_report

def final_model(clf):
    
    ## train classifier
    model = clf.fit(x_train, y_train)
    
    ## test on validation
    predicted = model.predict(x_valid)
    predicted_prob = model.predict_proba(x_valid)
    
    classes = np.unique(y_valid)
    y_valid_array = pd.get_dummies(y_valid, drop_first=False).values
    

    ## Accuracy, Precision, Recall

    accuracy = accuracy_score(y_valid, predicted)
    auc = roc_auc_score(y_valid, predicted_prob, multi_class="ovr")
    print("Accuracy:",  round(accuracy,2))
    print("Auc:", round(auc,2))
    print("Detail:")
    print(classification_report(y_valid, predicted))

    ## Plot confusion matrix
    cm = confusion_matrix(y_valid, predicted)
    fig, ax = plt.subplots()
    sns.heatmap(cm, annot=True, fmt='d', ax=ax, cmap=plt.cm.Blues, 
                cbar=False)
    ax.set(xlabel="Pred", ylabel="True", xticklabels=classes, 
           yticklabels=classes, title="Confusion matrix")
    plt.yticks(rotation=0)

    fig, ax = plt.subplots(1,2, figsize = (20,5))
    ## Plot roc
    for i in range(len(classes)):
        fpr, tpr, thresholds = metrics.roc_curve(y_valid_array[:,i], predicted_prob[:,i])
        ax[0].plot(fpr, tpr, lw=3, label='{0} (area={1:0.2f})'.format(classes[i], metrics.auc(fpr, tpr))
                   )
    ax[0].plot([0,1], [0,1], color='navy', lw=3, linestyle='--')
    ax[0].set(xlim=[-0.05,1.0], ylim=[0.0,1.05], 
              xlabel='False Positive Rate', ylabel="True Positive Rate (Recall)", 
              title="Receiver operating characteristic")
    ax[0].legend(loc="lower right")
    ax[0].grid(True)

    ## Plot precision-recall curve
    for i in range(len(classes)):
        precision, recall, thresholds = metrics.precision_recall_curve(
                     y_valid_array[:,i], predicted_prob[:,i])
        ax[1].plot(recall, precision, lw=3, 
                   label='{0} (area={1:0.2f})'.format(classes[i], 
                                      metrics.auc(recall, precision))
                  )
    ax[1].set(xlim=[0.0,1.05], ylim=[0.0,1.05], xlabel='Recall', 
              ylabel="Precision", title="Precision-Recall curve")
    ax[1].legend(loc="best")
    ax[1].grid(True)
    plt.show()
    
    return model


mnb = final_model(MultinomialNB())

feature_names = vectorizer.get_feature_names()
coefs_mnb = sorted(zip(feature_names, mnb.coef_[0]), key = lambda x: x[1], reverse=True)

fig, axs = plt.subplots(figsize=(20,4))
a = pd.DataFrame(coefs_mnb[:50])
a.set_index(0, inplace = True)
a.plot.bar(ax=axs)
axs.set_title('Top 50 Features for MultiNomial Naive Bayes')
plt.show()

Accuracy: 0.81
Auc: 0.96
Detail:
                  precision    recall  f1-score   support

    bank_service       0.79      0.76      0.78      4064
     credit_card       0.76      0.75      0.75      6069
credit_reporting       0.80      0.86      0.83     16050
 debt_collection       0.81      0.77      0.79     12127
            loan       0.79      0.72      0.75      6307
 money_transfers       0.88      0.57      0.69       946
        mortgage       0.89      0.92      0.91      8106

        accuracy                           0.81     53669
       macro avg       0.82      0.77      0.79     53669
    weighted avg       0.81      0.81      0.81     53669


lr = final_model(LogisticRegression(C=1, penalty='l2'))

feature_names = vectorizer.get_feature_names()
coefs_lr = sorted(zip(feature_names, lr.coef_[0]), key = lambda x: x[1], reverse=True)

fig, axs = plt.subplots(figsize=(20,4))
a = pd.DataFrame(coefs_lr[:50])
a.set_index(0, inplace = True)
a.plot.bar(ax=axs)
axs.set_title('Top 50 Features for Logistic Regression')
plt.show()

Accuracy: 0.85
Auc: 0.98
Detail:
                  precision    recall  f1-score   support

    bank_service       0.82      0.83      0.82      4064
     credit_card       0.82      0.80      0.81      6069
credit_reporting       0.86      0.86      0.86     16050
 debt_collection       0.83      0.84      0.83     12127
            loan       0.82      0.79      0.80      6307
 money_transfers       0.83      0.72      0.77       946
        mortgage       0.92      0.93      0.93      8106

        accuracy                           0.85     53669
       macro avg       0.84      0.83      0.83     53669
    weighted avg       0.85      0.85      0.85     53669


rf = final_model(RandomForestClassifier(n_estimators=200, max_depth=19, random_state=43))
importances = rf.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf.estimators_], axis=0)
forest_importances = pd.Series(importances, index=feature_names)

fig, axs = plt.subplots(figsize=(20,4))
pd.DataFrame(forest_importances).sort_values(by = 0,ascending=False)[:50].plot.bar(ax=axs)
ax.set_title("Feature importances using MDI")

Accuracy: 0.79
Auc: 0.96
Detail:
                  precision    recall  f1-score   support

    bank_service       0.80      0.71      0.76      4064
     credit_card       0.82      0.64      0.72      6069
credit_reporting       0.72      0.92      0.81     16050
 debt_collection       0.78      0.78      0.78     12127
            loan       0.87      0.63      0.73      6307
 money_transfers       1.00      0.17      0.30       946
        mortgage       0.90      0.90      0.90      8106

        accuracy                           0.79     53669
       macro avg       0.84      0.68      0.71     53669
    weighted avg       0.80      0.79      0.78     53669

Text(0.5, 1.0, 'Feature importances using MDI')


final_model(DecisionTreeClassifier())

Accuracy: 0.76
Auc: 0.83
Detail:
                  precision    recall  f1-score   support

    bank_service       0.66      0.66      0.66      4064
     credit_card       0.68      0.65      0.67      6069
credit_reporting       0.80      0.85      0.83     16050
 debt_collection       0.75      0.74      0.75     12127
            loan       0.70      0.66      0.68      6307
 money_transfers       0.60      0.58      0.59       946
        mortgage       0.85      0.84      0.84      8106

        accuracy                           0.76     53669
       macro avg       0.72      0.71      0.72     53669
    weighted avg       0.76      0.76      0.76     53669


"""
Out of all the models tested - Logistic Regression showed the most promising performance

Gradient Boosting model also showed high potential - However, due to limitation of time
GBM model could not be tuned to generate the best performance - this can be addressed in future work

Moreover, more advanced techniques using word-embedding and sequence model such as LSTM NN
can be tested in future. As an alternative, we tested the performance on a basic LSTM model and found
it to be very powerful!

However, based on the performance of the model using Logistic Regression 
we are quite comfortable. Moreover, logistic regression model provides better explainability 
and overall persimonious. Therefore, Logistic Regression model is the "Chamption Model" for this project
"""


"""
Thank you for taking the time to go over the work and I appreciate your time and considerations

"""

	complaint_id	length	num_words	masked_XX	caps_letter	caps_word	exclaimations	questions	punct	sp_char	unique_words	capsL_to_length	capsW_to_length	capsW_to_words	unique_to_words	masked_to_words
count	2.683430e+05	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000	268343.000000
mean	2.306798e+06	1081.071628	199.005765	10.281013	85.082339	21.716624	0.311743	0.216506	28.086352	3.774952	104.923538	0.082369	0.021188	0.114700	0.637097	0.052961
std	4.910311e+05	1149.470828	210.259267	19.972965	168.185135	37.327132	2.178775	1.248002	40.067409	10.433873	78.188712	0.109624	0.024142	0.132965	0.206245	0.065825
min	1.290155e+06	5.000000	1.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1.000000	0.000000	0.000000	0.000000	0.013021	0.000000
5%	1.418536e+06	131.000000	24.000000	0.000000	4.000000	1.000000	0.000000	0.000000	2.000000	0.000000	20.000000	0.014973	0.003654	0.020576	0.411658	0.000000
10%	1.539736e+06	203.000000	37.000000	0.000000	7.000000	2.000000	0.000000	0.000000	3.000000	0.000000	30.000000	0.020833	0.006462	0.035915	0.453757	0.000000
25%	1.907122e+06	392.000000	72.000000	2.000000	19.000000	6.000000	0.000000	0.000000	8.000000	0.000000	51.000000	0.034586	0.010909	0.059880	0.530211	0.014085
50%	2.413685e+06	746.000000	138.000000	5.000000	45.000000	13.000000	0.000000	0.000000	17.000000	1.000000	86.000000	0.056027	0.016279	0.088000	0.626374	0.036364
75%	2.738736e+06	1372.000000	254.000000	12.000000	95.000000	26.000000	0.000000	0.000000	35.000000	4.000000	137.000000	0.088246	0.023280	0.125000	0.734694	0.068966
90%	2.872482e+06	2363.000000	435.000000	24.000000	186.000000	47.000000	1.000000	1.000000	64.000000	10.000000	205.000000	0.140065	0.033939	0.181818	0.841270	0.115672
95%	2.919234e+06	3236.900000	594.000000	36.000000	277.000000	68.000000	2.000000	1.000000	89.000000	15.000000	258.000000	0.207716	0.047288	0.253731	0.909091	0.159664
99%	2.967017e+06	4757.000000	876.580000	77.000000	651.000000	145.000000	6.000000	4.000000	159.000000	33.000000	348.000000	0.781250	0.170940	0.965274	1.000000	0.330660
max	2.995824e+06	31735.000000	6320.000000	1811.000000	24835.000000	5644.000000	368.000000	342.000000	5229.000000	1518.000000	1839.000000	0.933333	0.260000	10.000000	74.000000	1.000000

	count	mean	std	min	25%	50%	75%	max
complaint_id	268343.0	2.306798e+06	491031.135984	1.290155e+06	1.907122e+06	2.413685e+06	2.738736e+06	2.995824e+06
length	268343.0	1.081072e+03	1149.470828	5.000000e+00	3.920000e+02	7.460000e+02	1.372000e+03	3.173500e+04
num_words	268343.0	1.990058e+02	210.259267	1.000000e+00	7.200000e+01	1.380000e+02	2.540000e+02	6.320000e+03
masked_XX	268343.0	1.028101e+01	19.972965	0.000000e+00	2.000000e+00	5.000000e+00	1.200000e+01	1.811000e+03
caps_letter	268343.0	8.508234e+01	168.185135	0.000000e+00	1.900000e+01	4.500000e+01	9.500000e+01	2.483500e+04
caps_word	268343.0	2.171662e+01	37.327132	0.000000e+00	6.000000e+00	1.300000e+01	2.600000e+01	5.644000e+03
exclaimations	268343.0	3.117428e-01	2.178775	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	3.680000e+02
questions	268343.0	2.165065e-01	1.248002	0.000000e+00	0.000000e+00	0.000000e+00	0.000000e+00	3.420000e+02
punct	268343.0	2.808635e+01	40.067409	0.000000e+00	8.000000e+00	1.700000e+01	3.500000e+01	5.229000e+03
sp_char	268343.0	3.774952e+00	10.433873	0.000000e+00	0.000000e+00	1.000000e+00	4.000000e+00	1.518000e+03
unique_words	268343.0	1.049235e+02	78.188712	1.000000e+00	5.100000e+01	8.600000e+01	1.370000e+02	1.839000e+03
capsL_to_length	268343.0	8.236949e-02	0.109624	0.000000e+00	3.458647e-02	5.602716e-02	8.824587e-02	9.333333e-01
capsW_to_length	268343.0	2.118846e-02	0.024142	0.000000e+00	1.090909e-02	1.627907e-02	2.327988e-02	2.600000e-01
capsW_to_words	268343.0	1.147000e-01	0.132965	0.000000e+00	5.988024e-02	8.800000e-02	1.250000e-01	1.000000e+01
unique_to_words	268343.0	6.370967e-01	0.206245	1.302083e-02	5.302106e-01	6.263736e-01	7.346939e-01	7.400000e+01
masked_to_words	268343.0	5.296059e-02	0.065825	0.000000e+00	1.408451e-02	3.636364e-02	6.896552e-02	1.000000e+00

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	xxxx xxxx	xxxx xxxx	xxxx xxxx	xxxx xxxx	xxxx xxxx	xxxx xxxx	xxxx xxxx
1	my account	xx xx	xx xx	xx xx	xx xx	xx xx	xx xx
2	xx xx	xx xxxx	my credit	my credit	xx xxxx	xx xxxx	xx xxxx
3	xx xxxx	credit card	xx xxxx	xx xxxx	of the	my account	of the
4	of the	my credit	credit report	of the	xxxx and	the money	xxxx and

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	xxxx xxxx xxxx	xxxx xxxx xxxx	xxxx xxxx xxxx	xxxx xxxx xxxx	xxxx xxxx xxxx	xxxx xxxx xxxx	xxxx xxxx xxxx
1	xx xx xxxx	xx xx xxxx	xx xx xxxx	xx xx xxxx	xx xx xxxx	xx xx xxxx	xx xx xxxx
2	on xxxx xxxx	on xx xx	my credit report	my credit report	on xx xx	on xx xx	on xxxx xxxx
3	on xx xx	on xxxx xxxx	on my credit	on my credit	on xxxx xxxx	on xxxx xxxx	on xx xx
4	bank of america	my credit card	xxxx xx xx	on xx xx	in xx xx	xxxx xxxx and	in xx xx

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	custimers	supiscious	l2inc	contraire	dibilitating	recounted	2321
1	ec	juiy	spooing	xxxx9	ques	becase	hma
2	ivite	resurfacing	inally	perfecly	originat	unexpired	discord
3	dividened	ureasonable	clc	passcodes	autodrafts	dtd	shiny
4	mustve	takeoff	dangers	remides	eps	iao	escrowthe

A. Exploratory Data Analysis¶

B. Text Processing¶

B. I. Before Text Cleaning - Just Exploring¶

B.II. Text Cleaning¶

B. II. After Text Cleaning¶

C. Model Building¶

C. II. Hyperparameter Tuning¶

C.III. Model Evaluation¶

D. Conclusion¶

Thank You¶

	complaint_id	product_group	text
0	2815595	bank_service	On XX/XX/2017 my check # XXXX was debited from my checking account, and this check wasn't cashed by XXXX XXXX XXXX, I requested a copy of the endorsement of whom the fund was credited to, Bank of America can't give me satisfactory answer, since this contractor breach of contract that I needed proof of payment in court.
1	2217937	bank_service	I opened a Bank of the the West account. The account came with a promotion. The promotion was get {$100.00} for setting up a qualifying direct deposit. I have received that promotion. The other promotion was, " get {$50.00} for making XXXX debit card purchases each month for 2 months. '' I have met the terms for the {$50.00} promotion and have not received the promotion that I am entitled to. I want the {$50.00} that I am entitled to that Bank of the West has not given me.

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	the	the	xxxx	xxxx	xxxx	xxxx	xxxx
1	xxxx	xxxx	the	the	the	the	the
2	to	to	to	to	to	to	to
3	and	and	and	and	and	and	and
4	my	my	my	my	my	my	my

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	wasn cashed	since however	three inaccurate	settle 4000	suspension since	main concern	on box
1	contractor breach	some 1500	month 840	official my	creditor 17	shows messages	npv property
2	purchases each	year computer	correctly requested	take asked	joining military	messages not	qualify see
3	nj opened	00 turned	giving response	evasive in	on orders	transfer pending	attached actual
4	early 2015	renew off	updated noting	correspondence re	orders assigning	pending am	actual appeal

	bank_service	credit_card	credit_reporting	debt_collection	loan	money_transfers	mortgage
0	this check wasn	paid them as	disputed three inaccurate	financial systems agreed	exactly sure who	com for buying	modification hamp xxxx
1	check wasn cashed	agreed monthly ever	three inaccurate 30	to settle 4000	time xxxx or	selling bitcoin my	mistake on box
2	wasn cashed by	ever since however	to experian submitted	settle 4000 00	xxxx or sallie	bitcoin my main	on box xxxx
3	of the endorsement	since however now	experian including record	4000 00 debt	mae my first	my main concern	my npv property
4	endorsement of whom	me credit any	each month 840	2000 00 if	first disbursement date	main concern is	npv property taxes

	PCA1	PCA2	class	color
0	-0.058879	-0.052114	bank_service	tab:blue
1	-0.084443	-0.109562	bank_service	tab:blue
2	-0.049013	-0.051678	bank_service	tab:blue
3	-0.1633	-0.104556	bank_service	tab:blue
4	-0.082661	-0.064461	bank_service	tab:blue

	Mean Accuracy	Standard deviation
model_name
DecisionTreeClassifier	0.762207	0.001802
LogisticRegression	0.846406	0.002326
MultinomialNB	0.809930	0.001934

	able	accidentally	account	account balance	account negative	action	admitted	affect	affect credit	afford	afford pay	ago	ahead	allowed	answer	answer question	anyways	attitude	attorney	authority	avoided	aware	aware xxxx	badly	balance	balance nt	bank	banker	believe	better	boa	bofa	borrowed	box	buy	ca	ca nt	calling	came	case	cash	certain	cfpb	check	check status	checking	child	collector	come	company	company xxxx	complaint	complaint submitted	complaint xxxx	confused	contacting	court	created	credit	credit told	crook	customer	customer service	daily	damaging	day	debt	debt collector	debt owe	deposit	described	didnt	direct	direct deposit	discussion	dispute	disputed	dollar	dropped	early	education	employment	entire	especially	expecting	family	fee	female	final	finally	finally got	financial	following	foreclosure	foreclosure action	frustrated	frustrating	fully	fully aware	fully paid	ga	going	got	granted	hearing	help	helpful	hoa	home	house	husband	illegal	immediately	informed	informing	initial	instructed	issue	ive	judge	know	know xxxx	later	le	leave	led	lending	letter	letter mail	like	like nt	link	list	listed	little	logged	logging	long	long time	lower	mail	making	making payment	manage	mention	mess	mid	monday	monday xxxx	money	month	month sent	needed	negative	new	new owner	nt	nt afford	nt going	obvious	ok	online	open	open account	order	outcome	overdraft	overdraft fee	owe	owe money	owed	owner	paid	paid money	particular	pay	pay pay	pay time	paying	payment	payment paid	phone	phone company	phone xxxx	point	predatory	predatory lending	prior	proceed	process	program	property	question	received	received account	received day	received letter	receiving	recovery	recovery solution	reflected	reflects	regarding	regarding xxxx	representative	representative xxxx	represented	representing	resolve	responded	response	result	resulted	resulting	retained	review	reviewed	reviewing	ridiculous	room	rude	ruined	ruling	saw	say	second	second time	seeing	sell	senior	sent	sent debt	service	site	site xxxx	solution	spoke	spoke representative	stated	status	steal	stolen	stopped	store	struggling	submitted	submitted complaint	subsequent	successful	sudden	sunday	switch	switched	theft	thief	think	time	time ago	time went	told	told month	traded	transaction	transpired	tried	tried dispute	true	trying	trying resolve	u	unable	understanding	update	value	value home	view	want	way	week	week later	went	went ahead	went court	wife	wife xxxx	wish	withdraw	withdrawal	work	worker	working	xxxx	xxxx account	xxxx attorney	xxxx balance	xxxx complaint	xxxx dollar	xxxx following	xxxx list	xxxx listed	xxxx long	xxxx month	xxxx paid	xxxx phone	xxxx store	xxxx xxxx	xxxx year	xxxxxxxx	xxxxxxxx xxxx	year	year ago
196350	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.170615	0.238785	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.105374	0.26876	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.372042	0.00000	0.000000	0.143474	0.155929	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.213623	0.117874	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.161951	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.191043	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.131188	0.172213	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.156031	0.234361	0.261203	0.000000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.093331	0.000000	0.000000	0.091086	0.000000	0.221845	0.000000	0.077253	0.208445	0.000000	0.000000	0.000000	0.000000	0.199889	0.24133	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.19854	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.22245	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.123233	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.112602	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
18866	0.000000	0.00000	0.091026	0.119579	0.140537	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.123045	0.000000	0.087582	0.13141	0.000000	0.000000	0.000000	0.000000	0.157743	0.000000	0.000000	0.000000	0.135627	0.00000	0.146464	0.132179	0.079903	0.10929	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.082304	0.000000	0.000000	0.000000	0.000000	0.088827	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.129729	0.000000	0.000000	0.118677	0.000000	0.000000	0.000000	0.070453	0.083399	0.000000	0.000000	0.052523	0.000000	0.000000	0.000000	0.096728	0.00000	0.000000	0.10269	0.130595	0.000000	0.000000	0.00000	0.154151	0.000000	0.000000	0.000000	0.112754	0.000000	0.116087	0.148465	0.000000	0.143512	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.125227	0.131005	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.065535	0.130787	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.065983	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.067513	0.165355	0.000000	0.000000	0.000000	0.187181	0.000000	0.160619	0.000000	0.000000	0.102776	0.000000	0.000000	0.000000	0.141598	0.000000	0.000000	0.148546	0.117265	0.144877	0.063212	0.000000	0.0000	0.000000	0.086611	0.000000	0.000000	0.056519	0.000000	0.000000	0.000000	0.123308	0.0787	0.085128	0.116766	0.000000	0.000000	0.11326	0.123266	0.000000	0.000000	0.000000	0.000000	0.057240	0.163336	0.000000	0.000000	0.000000	0.000000	0.077204	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.087982	0.048832	0.162547	0.000000	0.000000	0.000000	0.000000	0.000000	0.128223	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.085642	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.090961	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.061795	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.00000	0.136603	0.00000	0.143562	0.0000	0.000000	0.000000	0.000000	0.095059	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.086395	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.125311	0.000000	0.000000	0.000000	0.000000	0.071993	0.149834	0.000000	0.000000	0.000000	0.000000	0.125358	0.123394	0.000000	0.000000	0.000000	0.085013	0.079309	0.000000	0.111401	0.000000	0.180959	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.080347	0.000000	0.000000	0.000000	0.000000	0.000000
263855	0.040009	0.08682	0.000000	0.000000	0.000000	0.044994	0.068334	0.000000	0.000000	0.000000	0.000000	0.074401	0.000000	0.108297	0.000000	0.00000	0.000000	0.000000	0.130705	0.065831	0.000000	0.086438	0.088588	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.071915	0.194477	0.073329	0.00000	0.000000	0.000000	0.000000	0.000000	0.049133	0.123527	0.092477	0.000000	0.000000	0.039389	0.08035	0.046904	0.063179	0.000000	0.000000	0.000000	0.000000	0.0943	0.091044	0.099651	0.000000	0.000000	0.096415	0.000000	0.000000	0.000000	0.231763	0.000000	0.000000	0.000000	0.069505	0.000000	0.000000	0.000000	0.000000	0.000000	0.07308	0.000000	0.00000	0.000000	0.078034	0.062724	0.04209	0.000000	0.103638	0.058835	0.065457	0.000000	0.095358	0.000000	0.000000	0.051388	0.000000	0.08323	0.056695	0.047507	0.075611	0.041236	0.076817	0.050612	0.087631	0.000000	0.000000	0.106613	0.091618	0.091729	0.000000	0.039337	0.040422	0.069613	0.144902	0.000000	0.000000	0.253689	0.088811	0.000000	0.000000	0.000000	0.041677	0.042391	0.11036	0.057994	0.000000	0.035903	0.000000	0.180489	0.035689	0.07298	0.000000	0.053491	0.000000	0.070539	0.000000	0.000000	0.000000	0.000000	0.000000	0.070696	0.097236	0.076495	0.000000	0.069074	0.000000	0.085422	0.072041	0.000000	0.000000	0.000000	0.000000	0.000000	0.064872	0.075478	0.000000	0.000000	0.000000	0.034395	0.000000	0.0000	0.044475	0.000000	0.063966	0.158649	0.000000	0.000000	0.000000	0.074659	0.000000	0.0000	0.000000	0.000000	0.000000	0.128427	0.00000	0.000000	0.000000	0.000000	0.044486	0.124302	0.052735	0.000000	0.071054	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.043281	0.000000	0.00000	0.081491	0.000000	0.041034	0.000000	0.047451	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.042534	0.077145	0.000000	0.000000	0.077341	0.073575	0.000000	0.088803	0.069543	0.000000	0.109316	0.068469	0.078596	0.050216	0.059344	0.064437	0.00000	0.075007	0.00000	0.073337	0.090397	0.058124	0.041351	0.049494	0.07149	0.070983	0.060337	0.074641	0.000000	0.000000	0.000000	0.10272	0.092376	0.000000	0.000000	0.000000	0.038655	0.082081	0.128626	0.099952	0.058594	0.000000	0.00000	0.075367	0.076729	0.12367	0.000000	0.07721	0.000000	0.0000	0.000000	0.048979	0.077112	0.000000	0.050973	0.088196	0.087118	0.027843	0.000000	0.000000	0.000000	0.088334	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.047168	0.061901	0.05364	0.060719	0.087631	0.000000	0.000000	0.087766	0.000000	0.000000	0.039174	0.000000	0.086943	0.000000	0.000000	0.066291	0.000000	0.000000	0.040274	0.000000	0.049913	0.050420	0.000000	0.074712	0.000000	0.071999	0.000000	0.079527	0.090564	0.074415	0.000000	0.000000	0.000000	0.000000	0.000000	0.031020	0.050672	0.049662	0.045337	0.029927	0.052273
156407	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.332207	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.000000	0.222683	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.258411	0.000000	0.000000	0.120779	0.000000	0.295762	0.000000	0.00000	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.216788	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.240673	0.000000	0.000000	0.119661	0.285766	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.161388	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.138683	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.167756	0.270683	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.107135	0.000000	0.000000	0.194421	0.000000	0.231112	0.360148	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.255737	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.0000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.160694	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
139840	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.120066	0.137675	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.161441	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.159081	0.000000	0.00000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.26616	0.105696	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.112095	0.000000	0.000000	0.00000	0.000000	0.000000	0.094316	0.087877	0.047989	0.097024	0.0000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.082427	0.153214	0.000000	0.000000	0.000000	0.000000	0.000000	0.051370	0.053843	0.100632	0.000000	0.000000	0.00000	0.099435	0.00000	0.000000	0.000000	0.066589	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.085408	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.116468	0.000000	0.104793	0.000000	0.000000	0.00000	0.073905	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.090685	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.090507	0.1578	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.0000	0.000000	0.000000	0.080885	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.055984	0.000000	0.000000	0.114664	0.154445	0.000000	0.000000	0.000000	0.000000	0.161321	0.150367	0.105796	0.000000	0.000000	0.00000	0.000000	0.122261	0.000000	0.162153	0.000000	0.000000	0.047760	0.000000	0.152051	0.000000	0.087983	0.000000	0.000000	0.000000	0.138802	0.000000	0.000000	0.074257	0.117153	0.000000	0.000000	0.085274	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.00000	0.000000	0.10469	0.000000	0.000000	0.000000	0.000000	0.186702	0.00000	0.000000	0.000000	0.000000	0.089086	0.147885	0.000000	0.00000	0.000000	0.000000	0.073354	0.124871	0.000000	0.000000	0.000000	0.000000	0.000000	0.110233	0.00000	0.000000	0.000000	0.00000	0.000000	0.00000	0.000000	0.1308	0.134213	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.084739	0.158284	0.158284	0.000000	0.000000	0.072344	0.151439	0.102785	0.070267	0.140688	0.000000	0.000000	0.000000	0.00000	0.000000	0.000000	0.000000	0.000000	0.000000	0.073194	0.113612	0.000000	0.000000	0.000000	0.098442	0.138108	0.000000	0.000000	0.000000	0.000000	0.144944	0.000000	0.090629	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.161612	0.095331	0.102977	0.109761	0.143847	0.085932	0.000000	0.000000	0.000000	0.000000	0.000000